package com.aiolos.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.util.Progressable;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.net.URI;

/**
 * 使用JAVA API操作HDFS文件系统实例
 */
public class HDFSAPIDemo02 {
    //HDFS的路径,core-site.xml中配置的端口号
    private final String HDFS_PATH = "hdfs://192.168.126.129:8020";
    private FileSystem fileSystem = null;
    private Configuration configuration = null;

    /**
       单元测试之前的准备工作,准备环境,加载配置
     */
    @Before
    public void setUp() throws Exception{
        System.out.println("HDFSAPIDemo02 is setUp.....");
        configuration = new Configuration();
        //设置默认副本系数为1
        configuration.set("dfs.replication","1");
        URI uri = new URI(HDFS_PATH);
        //构造一个访问指定HDFS系统的客户端对象
        //第一个参数：指定HDFS系统的URI
        //第二个参数：客户端指定的配置参数对象
        //第三个参数：客户端的身份，即用户名
        fileSystem = FileSystem.get(uri,configuration,"hadoop");
    }

    /**
       单元测试之后的工作,清理环境,释放资源等等
     */
    @After
    public void tearDown(){
        configuration = null;
        fileSystem = null;
        System.out.println("HDFSAPIDemo02 is tearDown....");
    }

    /**
     * 在HDFS中创建文件夹
     * @throws Exception
     */
    @Test
    public void mkdir() throws Exception{
        boolean result = fileSystem.mkdirs(new Path("/hadoopapi/test/"));
        System.out.println(result);
    }

    /**
     * 查看HDFS文件中的内容
     */
    @Test
    public void text() throws Exception{
        FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"),2048);
        IOUtils.copyBytes(in,System.out,1024);
    }

    /**
     * 在HDFS中创建文件并写入内容
     */
    @Test
    public void create() throws Exception{
        //创建文件
        FSDataOutputStream out = fileSystem.create(new Path("/hadoopapi/test/a.txt"));
        //向文件中写入内容
        out.writeUTF("Hello PK");
        out.flush();
        out.close();
    }

    /**
     * 获取Java API默认的副本系数
     */
    @Test
    public void getReplication(){
        System.out.println(configuration.get("dfs.replication"));
    }

    /**
     * 文件重命名
     */
    @Test
    public void rename() throws Exception{
        Path oldPath = new Path("/hadoopapi/test/a.txt");
        Path newPath = new Path("/hadoopapi/test/b.txt");
        boolean result = fileSystem.rename(oldPath,newPath);
        System.out.println(result);
    }

    /**
     * copyFromLocalFile
     * 复制本地文件到HDFS中
     * java.io.FileNotFoundException: File /home/hadoop/temp/hello.txt does not exist
     */
    @Test
    public void copyFromLocalFile() throws Exception{
        Path localPath = new Path("/home/hadoop/temp/hello.txt");
        Path hdfsPath = new Path("/hadoopapi/test/");
        fileSystem.copyFromLocalFile(localPath,hdfsPath);
    }

    /**
     * 复制本地大文件到HDFS中,带进度条
     */
    @Test
    public void copyFromLocalBigFile() throws Exception{

        InputStream in = new BufferedInputStream(new FileInputStream(new File(("/home/hadoop/software/hive-1.1.0-cdh5.15.1.tar.gz"))));
        Path hdfsPath = new Path("/hadoopapi/test/hive-1.1.0-cdh5.15.1.tar.gz");
        FSDataOutputStream out = fileSystem.create(hdfsPath,
                new Progressable() {
                    @Override
                    public void progress() {
                        //进度条
                        System.out.print(".");
                    }
                });


        IOUtils.copyBytes(in,out,4096);
    }

    /**
     * 下载HDFS文件到本地
     */
    @Test
    public void copyToLocalFile() throws Exception{

        Path hdfsPath = new Path("/hadoopapi/test/b.txt");
        Path localPath = new Path("/home/hadoop/temp/");
        fileSystem.copyToLocalFile(hdfsPath,localPath);
    }

    /**
     * 列出HDFS下文件夹下的所有内容
     */
    @Test
    public void listFile() throws Exception{
        FileStatus[] fileStatuses = fileSystem.listStatus(new Path("/hadoopapi/test/")) ;
        for (FileStatus file:fileStatuses) {
            String isDir = file.isDirectory()?"文件夹":"文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();
            System.out.println(isDir + "\t"+permission+"\t"
                    + replication + "\t" + length +"\t"
                    + path);
        }
    }

    /**
     * 递归列出HDFS下文件夹下的所有文件
     */
    @Test
    public void listFileRecursive() throws Exception{
        RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hadoopapi/test/"),true) ;
        while (files.hasNext()){
            LocatedFileStatus file = files.next();
            String isDir = file.isDirectory()?"文件夹":"文件";
            String permission = file.getPermission().toString();
            short replication = file.getReplication();
            long length = file.getLen();
            String path = file.getPath().toString();
            System.out.println(isDir + "\t"+permission+"\t"
                    + replication + "\t" + length +"\t"
                    + path);
        }
    }

    /**
     * 查看文件块信息
     */
    @Test
    public void getFileBlockLocations() throws Exception{
        FileStatus file = fileSystem.getFileStatus(new Path("/hadoopapi/test/b.txt"));
        BlockLocation[] blocks = fileSystem.getFileBlockLocations(file,0,file.getLen());
        for (BlockLocation block:blocks) {
            for(String name:block.getNames()){
                System.out.println(name +" : "+ block.getOffset() + " : " + block.getLength());
            }
        }
    }

    /**
     * 删除HDFS中的文件
     */
    @Test
    public void deleteFile() throws Exception{
        Path delFile = new Path("/hadoopapi/test/b.txt");

        boolean result = fileSystem.delete(delFile,true);

        System.out.println(result);
    }
}
